'''
These are some examples of reading in data and doing some basic statistics.
'''

import numpy as np
import os
import numpy.lib.recfunctions as nprf

# must have the directory containing scipystats added to PYTHONPATH
# the file structure will obviously change in the future
from scipystats.sandbox.string2dummy import string2dummy as s2d
from scipystats.sandbox.load_dataset import load_dataset

loc='http://eagle1.american.edu/~js2796a/data/handguns_data.csv'
relpath=load_dataset(loc)
dta=np.recfromcsv(relpath)
dta_ns=s2d(dta)
narr=np.column_stack(dta_ns[col] for col in dta_ns.dtype.names)
narr.mean(0)
narr.var(0)
# I wonder if this can't be achieved using masks.

# This is the dtype
print dta.dtype
# All our variables names
print dta.dtype.names
# The first column of this data
print dta['year']
# or, much prettier
print dta.year
# You can refer to the data by field number if you so desire
print dta.field(0) # where the first field is the year

# Correlation matrix
print np.corrcoef((dta['shall'],dta['mur'],dta['stpop']))
print np.corrcoef((dta.shall, dta.mur, dta.stpop))

# If the file is not comma-separated then you would use recfromtxt and specify a delimiter
# Also recfromtxt has no headers by default
# This returns the same as above
#dta2=np.recfromtxt(relpath,delimiter=',',names=True)
#dta2_ns=string2dummy(dta2)
#np.testing.assert_array_equal(dta_ns,dta2_ns)


# Now let's do a basic OLS example with our record array
# You must have the Ols.py example in your current working directory (or Python path I suppose)
# Found here: http://code.google.com/p/joepython/source/browse/trunk/joepython/scipystats/examples/regression/olsexample.py

from os import getcwd
try: import Ols; have_ols=True
except ImportError: print "Ols.py not found in "+getcwd(); have_ols=False
if have_ols:
# The user will make a list of varnames.  Since record arrays can have field names (our variable names)
# or "titles," which could act like data labels I believe, this list could contain either
# However, if dtype=None as is the default in recfrom*, so that the type is the same as the data
# the dtype is the same as the first example under 4) here
# http://docs.scipy.org/doc/numpy/user/basics.rec.html

    # the user would have to specify the varnames list
    # I think this is all the user should include in the function call
    varnames=['shall','incarc_rate','density','avginc','pop','pb1064','pw1064','pm1029']
    # then in the __init__ we would have something like 
    x=np.column_stack((dta[col] for col in varnames))
    OLS_example_one=Ols.ols(np.log(dta.vio),x,y_varnm='log_vio',x_varnm=varnames)
    # Ideally (with record arrays), this will simply be something like
    # OLS_example_one=Ols.ols(y,varnames)
    # Where y is the dependent variable name and x is a list of independent variables
    OLS_example_one.summary()
